{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read Data Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:28:57.708241Z",
     "start_time": "2017-06-03T13:28:57.293983Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "from collections import namedtuple\n",
    "pd.set_option(\"display.max_rows\",35)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:28:57.780946Z",
     "start_time": "2017-06-03T13:28:57.710081Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class dataset:\n",
    "    kdd_train_2labels = pd.read_pickle(\"dataset/kdd_train_2labels_20percent.pkl\")\n",
    "    kdd_test_2labels = pd.read_pickle(\"dataset/kdd_test_2labels_20percent.pkl\")\n",
    "    \n",
    "    kdd_train_5labels = pd.read_pickle(\"dataset/kdd_train_5labels.pkl\")\n",
    "    kdd_test_5labels = pd.read_pickle(\"dataset/kdd_test_5labels.pkl\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:28:57.787237Z",
     "start_time": "2017-06-03T13:28:57.782601Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25192, 120)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_train_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:28:57.809278Z",
     "start_time": "2017-06-03T13:28:57.788638Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11850, 120)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_test_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:28:58.186981Z",
     "start_time": "2017-06-03T13:28:57.810637Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99148920692946862"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import model_selection as ms\n",
    "from sklearn import preprocessing as pp\n",
    "\n",
    "class preprocess:\n",
    "    \n",
    "    output_columns_2labels = ['is_Attack','is_Normal']\n",
    "    \n",
    "    x_input = dataset.kdd_train_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_output = dataset.kdd_train_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    x_test_input = dataset.kdd_test_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test = dataset.kdd_test_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    ss = pp.StandardScaler()\n",
    "\n",
    "    x_train = ss.fit_transform(x_input)\n",
    "    x_test = ss.transform(x_test_input)\n",
    "\n",
    "    y_train = y_output.values\n",
    "    y_test = y_test.values\n",
    "\n",
    "preprocess.x_train.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:29:14.345023Z",
     "start_time": "2017-06-03T13:28:58.188594Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import basic_rnn_seq2seq\n",
    "from tensorflow.contrib.rnn import RNNCell, LSTMCell, MultiRNNCell\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:29:14.527023Z",
     "start_time": "2017-06-03T13:29:14.346914Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class network(object):\n",
    "    \n",
    "    input_dim = 118\n",
    "    classes = 2\n",
    "    hidden_encoder_dim = 118\n",
    "    hidden_layers = 1\n",
    "    latent_dim = 10\n",
    "\n",
    "    hidden_decoder_dim = 118\n",
    "    lam = 0.01\n",
    "    \n",
    "    def __init__(self, classes, hidden_layers, num_of_features):\n",
    "        self.classes = classes\n",
    "        self.hidden_layers = hidden_layers\n",
    "        self.latent_dim = num_of_features\n",
    "            \n",
    "    def build_layers(self):\n",
    "        tf.reset_default_graph()\n",
    "        #learning_rate = tf.Variable(initial_value=0.001)\n",
    "\n",
    "        input_dim = self.input_dim\n",
    "        classes = self.classes\n",
    "        hidden_encoder_dim = self.hidden_encoder_dim\n",
    "        hidden_layers = self.hidden_layers\n",
    "        latent_dim = self.latent_dim\n",
    "        hidden_decoder_dim = self.hidden_decoder_dim\n",
    "        lam = self.lam\n",
    "        \n",
    "        with tf.variable_scope(\"Input\"):\n",
    "            self.x_input = tf.placeholder(\"float\", shape=[None, 1, input_dim])\n",
    "            self.y_input_ = tf.placeholder(\"float\", shape=[None, 1, classes])\n",
    "            self.keep_prob = tf.placeholder(\"float\")\n",
    "            self.lr = tf.placeholder(\"float\")\n",
    "            self.x_list = tf.unstack(self.x_input, axis= 1)\n",
    "            self.y_list_ = tf.unstack(self.y_input_, axis = 1)\n",
    "            self.y_ = self.y_list_[0]\n",
    "            \n",
    "            #GO = tf.fill((tf.shape(self.x)[0], 1), 0.5)\n",
    "            \n",
    "            #y_with_GO = tf.stack([self.y_, GO])\n",
    "            \n",
    "        with tf.variable_scope(\"lstm\"):\n",
    "            multi_cell = MultiRNNCell([LSTMCell(input_dim) for i in range(hidden_layers)] )\n",
    "            \n",
    "            self.y, states = basic_rnn_seq2seq(self.x_list, self.y_list_, multi_cell)\n",
    "            #self.y = tf.slice(self.y, [0, 0], [-1,2])\n",
    "            \n",
    "            #self.out = tf.squeeze(self.y)\n",
    "            \n",
    "            self.y = tf.layers.dense(self.y[0], classes, activation = tf.nn.softmax)\n",
    "        with tf.variable_scope(\"Loss\"):\n",
    "            \n",
    "            self.regularized_loss = tf.losses.mean_squared_error(self.y_, self.y)\n",
    "            correct_prediction = tf.equal(tf.argmax(self.y_, 1), tf.argmax(self.y, 1))\n",
    "            self.tf_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name = \"Accuracy\")\n",
    "\n",
    "        with tf.variable_scope(\"Optimizer\"):\n",
    "            learning_rate=self.lr\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            gradients, variables = zip(*optimizer.compute_gradients(self.regularized_loss))\n",
    "            gradients = [\n",
    "                None if gradient is None else tf.clip_by_value(gradient, -1, 1)\n",
    "                for gradient in gradients]\n",
    "            self.train_op = optimizer.apply_gradients(zip(gradients, variables))\n",
    "            #self.train_op = optimizer.minimize(self.regularized_loss)\n",
    "            \n",
    "        # add op for merging summary\n",
    "        #self.summary_op = tf.summary.merge_all()\n",
    "        self.pred = tf.argmax(self.y, axis = 1)\n",
    "        self.actual = tf.argmax(self.y_, axis = 1)\n",
    "\n",
    "        # add Saver ops\n",
    "        self.saver = tf.train.Saver()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T00:59:00.684124Z",
     "start_time": "2017-06-01T00:58:59.843181Z"
    }
   },
   "source": [
    "batch_iterations = 200\n",
    "\n",
    "x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, \n",
    "                                                                          preprocess.y_train, \n",
    "                                                                          test_size=0.1)\n",
    "batch_indices = np.array_split(np.arange(x_train.shape[0]), \n",
    "                                           batch_iterations)\n",
    "                                                                          \n",
    "for i in batch_indices:\n",
    "    print(x_train[i,np.newaxis,:])\n",
    "    print(y_train[i,np.newaxis,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:29:14.725088Z",
     "start_time": "2017-06-03T13:29:14.528685Z"
    },
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "class Train:    \n",
    "    \n",
    "    result = namedtuple(\"score\", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score_20', 'time_taken'])\n",
    "\n",
    "    predictions = {}\n",
    "\n",
    "    results = []\n",
    "    best_acc = 0\n",
    "    best_acc_global = 0\n",
    "\n",
    "    def train(epochs, net, h,f, lrs):\n",
    "        batch_iterations = 200\n",
    "        train_loss = None\n",
    "        Train.best_acc = 0\n",
    "        os.makedirs(\"dataset/tf_lstm_nsl_kdd-/hidden layers_{}_features count_{}\".format(h,f),\n",
    "                    exist_ok = True)\n",
    "        with tf.Session() as sess:\n",
    "            #summary_writer_train = tf.summary.FileWriter('./logs/kdd/VAE/training', graph=sess.graph)\n",
    "            #summary_writer_valid = tf.summary.FileWriter('./logs/kdd/VAE/validation')\n",
    "\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            start_time = time.perf_counter()\n",
    "            for lr in lrs:\n",
    "                for epoch in range(1, (epochs+1)):\n",
    "                    x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, \n",
    "                                                                              preprocess.y_train, \n",
    "                                                                              test_size=0.1)\n",
    "                    batch_indices = np.array_split(np.arange(x_train.shape[0]), \n",
    "                                               batch_iterations)\n",
    "\n",
    "                    for i in batch_indices:\n",
    "\n",
    "                        _, train_loss = sess.run([net.train_op, net.regularized_loss], #net.summary_op\n",
    "                                                              feed_dict={net.x_input: x_train[i,np.newaxis,:], \n",
    "                                                                         net.y_input_: y_train[i,np.newaxis,:], \n",
    "                                                                         net.keep_prob:1, net.lr:lr})\n",
    "                        #summary_writer_train.add_summary(summary_str, epoch)\n",
    "                        if(train_loss > 1e9):\n",
    "                            print(\"Step {} | Training Loss: {:.6f}\".format(epoch, train_loss))\n",
    "\n",
    "\n",
    "                    valid_accuracy,valid_loss = sess.run([net.tf_accuracy, net.regularized_loss], #net.summary_op \n",
    "                                                          feed_dict={net.x_input: x_valid[:,np.newaxis,:], \n",
    "                                                                     net.y_input_: y_valid[:,np.newaxis,:], \n",
    "                                                                     net.keep_prob:1, net.lr:lr})\n",
    "                    #summary_writer_valid.add_summary(summary_str, epoch)\n",
    "\n",
    "\n",
    "\n",
    "                    accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], \n",
    "                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], \n",
    "                                                                             net.keep_prob:1, net.lr:lr})\n",
    "\n",
    "                    print(\"Step {} | Training Loss: {:.6f} | Train Accuracy: {:.6f} | Test Accuracy: {:.6f}\".format(epoch, train_loss, valid_accuracy, accuracy))\n",
    "\n",
    "                    if accuracy > Train.best_acc_global:\n",
    "                                Train.best_acc_global = accuracy\n",
    "                                Train.pred_value = pred_value\n",
    "                                Train.actual_value = actual_value\n",
    "                                Train.best_parameters = \"Hidden Layers:{}, Features Count:{}\".format(h, f)\n",
    "\n",
    "                    if accuracy > Train.best_acc:\n",
    "\n",
    "                        #net.saver.save(sess, \"dataset/tf_vae_only_nsl_kdd_hidden layers_{}_features count_{}\".format(epochs,h,f))\n",
    "                        #Train.results.append(Train.result(epochs, f, h,valid_accuracy, accuracy))\n",
    "                        #curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,-2], \"Normal_prob\":y_pred[:, -1]})\n",
    "                        #Train.predictions.update({\"{}_{}_{}\".format(epochs,f,h):curr_pred})\n",
    "\n",
    "                        Train.best_acc = accuracy\n",
    "                        if not (np.isnan(train_loss)):\n",
    "                            net.saver.save(sess, \n",
    "                                       \"dataset/tf_lstm_nsl_kdd-/hidden layers_{}_features count_{}/model\"\n",
    "                                       .format(h,f), \n",
    "                                       global_step = epoch, \n",
    "                                       write_meta_graph=False)\n",
    "\n",
    "                        curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,-2], \"Normal_prob\":y_pred[:, -1], \"Prediction\":pred_value})\n",
    "                        Train.predictions.update({\"{}_{}_{}\".format(epochs*len(lrs),f,h):\n",
    "                                                  (curr_pred, \n",
    "                                                   Train.result(epochs*len(lrs), f, h,valid_accuracy, accuracy, time.perf_counter() - start_time))})\n",
    "\n",
    "\n",
    "\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:31:22.254361Z",
     "start_time": "2017-06-03T13:29:14.726799Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Layer Attributes - epochs:5 hidden layers:2 features count:122\n",
      "Step 1 | Training Loss: 0.008756 | Train Accuracy: 1.000000 | Test Accuracy: 0.966751\n",
      "Step 2 | Training Loss: 0.000002 | Train Accuracy: 0.999603 | Test Accuracy: 0.941688\n",
      "Step 3 | Training Loss: 0.000002 | Train Accuracy: 1.000000 | Test Accuracy: 0.948692\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.998413 | Test Accuracy: 0.996709\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.994684\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.994852\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.994852\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.994852\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.994852\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.994852\n",
      "Current Layer Attributes - epochs:5 hidden layers:4 features count:122\n",
      "Step 1 | Training Loss: 0.079646 | Train Accuracy: 0.942460 | Test Accuracy: 0.488354\n",
      "Step 2 | Training Loss: 0.014094 | Train Accuracy: 0.919444 | Test Accuracy: 0.538650\n",
      "Step 3 | Training Loss: 0.000388 | Train Accuracy: 0.990873 | Test Accuracy: 0.761435\n",
      "Step 4 | Training Loss: 0.000023 | Train Accuracy: 0.990873 | Test Accuracy: 0.709789\n",
      "Step 5 | Training Loss: 0.000059 | Train Accuracy: 0.992063 | Test Accuracy: 0.764051\n",
      "Step 1 | Training Loss: 0.017268 | Train Accuracy: 0.991667 | Test Accuracy: 0.764219\n",
      "Step 2 | Training Loss: 0.000093 | Train Accuracy: 0.989286 | Test Accuracy: 0.764304\n",
      "Step 3 | Training Loss: 0.008610 | Train Accuracy: 0.992063 | Test Accuracy: 0.764557\n",
      "Step 4 | Training Loss: 0.008566 | Train Accuracy: 0.990873 | Test Accuracy: 0.765316\n",
      "Step 5 | Training Loss: 0.008485 | Train Accuracy: 0.994048 | Test Accuracy: 0.766498\n",
      "Current Layer Attributes - epochs:5 hidden layers:6 features count:122\n",
      "Step 1 | Training Loss: 0.079746 | Train Accuracy: 0.946429 | Test Accuracy: 0.752068\n",
      "Step 2 | Training Loss: 0.079646 | Train Accuracy: 0.909921 | Test Accuracy: 0.767595\n",
      "Step 3 | Training Loss: 0.530973 | Train Accuracy: 0.457540 | Test Accuracy: 0.817553\n",
      "Step 4 | Training Loss: 0.566372 | Train Accuracy: 0.490079 | Test Accuracy: 0.808861\n",
      "Step 5 | Training Loss: 0.584071 | Train Accuracy: 0.465873 | Test Accuracy: 0.809536\n",
      "Step 1 | Training Loss: 0.504425 | Train Accuracy: 0.454762 | Test Accuracy: 0.809536\n",
      "Step 2 | Training Loss: 0.522124 | Train Accuracy: 0.472222 | Test Accuracy: 0.809536\n",
      "Step 3 | Training Loss: 0.539823 | Train Accuracy: 0.457143 | Test Accuracy: 0.809536\n",
      "Step 4 | Training Loss: 0.477876 | Train Accuracy: 0.472222 | Test Accuracy: 0.809620\n",
      "Step 5 | Training Loss: 0.522124 | Train Accuracy: 0.466667 | Test Accuracy: 0.809620\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "class Hyperparameters:\n",
    "#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]\n",
    "#    hidden_layers_arr = [2, 4, 6, 10]\n",
    "    features_arr = [122] #[4, 8, 16, 32]\n",
    "    hidden_layers_arr = [2, 4, 6]\n",
    "\n",
    "    epochs = [5]\n",
    "    lrs = [1e-2, 1e-4]\n",
    "    \n",
    "    for e, h, f in itertools.product(epochs, hidden_layers_arr, features_arr):\n",
    "        print(\"Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\".format(e,h,f))\n",
    "        n = network(2,h,f)\n",
    "        n.build_layers()\n",
    "        Train.train(e, n, h,f, lrs)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:31:22.261333Z",
     "start_time": "2017-06-03T13:31:22.256299Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dict1 = {}\n",
    "dict2 = []\n",
    "for k, (v1, v2) in Train.predictions.items():\n",
    "    dict1.update({k: v1})\n",
    "    dict2.append(v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:31:22.267703Z",
     "start_time": "2017-06-03T13:31:22.263473Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Train.predictions = dict1\n",
    "Train.results = dict2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:31:22.321044Z",
     "start_time": "2017-06-03T13:31:22.269372Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_results = pd.DataFrame(Train.results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:31:22.337050Z",
     "start_time": "2017-06-03T13:31:22.323187Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>122</td>\n",
       "      <td>2</td>\n",
       "      <td>0.998413</td>\n",
       "      <td>0.996709</td>\n",
       "      <td>8.649876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10</td>\n",
       "      <td>122</td>\n",
       "      <td>6</td>\n",
       "      <td>0.457540</td>\n",
       "      <td>0.817553</td>\n",
       "      <td>17.537749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>122</td>\n",
       "      <td>4</td>\n",
       "      <td>0.994048</td>\n",
       "      <td>0.766498</td>\n",
       "      <td>39.551998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  no_of_features  hidden_layers  train_score  test_score_20  \\\n",
       "0     10             122              2     0.998413       0.996709   \n",
       "2     10             122              6     0.457540       0.817553   \n",
       "1     10             122              4     0.994048       0.766498   \n",
       "\n",
       "   time_taken  \n",
       "0    8.649876  \n",
       "2   17.537749  \n",
       "1   39.551998  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.sort_values(by = 'test_score_20', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:31:22.346200Z",
     "start_time": "2017-06-03T13:31:22.338956Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(Train.predictions).to_pickle(\"dataset/tf_lstm_nsl_kdd_predictions-.pkl\")\n",
    "df_results.to_pickle(\"dataset/tf_lstm_nsl_kdd_scores-.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:31:22.427195Z",
     "start_time": "2017-06-03T13:31:22.348322Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    np.set_printoptions(precision=4)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j].round(4),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "def plot(actual_value, pred_value):\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    cm_2labels = confusion_matrix(y_pred = pred_value, y_true = actual_value)\n",
    "    plt.figure(figsize=[6,6])\n",
    "    plot_confusion_matrix(cm_2labels, preprocess.output_columns_2labels, normalize = True,\n",
    "                         title = Train.best_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:31:23.775641Z",
     "start_time": "2017-06-03T13:31:22.429165Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normalized confusion matrix\n",
      "[[ 0.9969  0.0031]\n",
      " [ 0.0042  0.9958]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAGgCAYAAAAtsfn1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XecXWW59vHfNZPeE2oyCQQILUEgJJQXFDmCEqRaKEoT\nEARBRERFBQUPEQ5iR0SKhKJ0hVBCkSNHpARCaAkIhJpMAiRAQhopM/f7x3pm2BkyLdkzs2ev65vP\n+sxe/Vl7Jvve972etZYiAjMzs3JQ0dENMDMzKxYHNTMzKxsOamZmVjYc1MzMrGw4qJmZWdlwUDMz\ns7LhoGZmZmXDQc3MzMqGg5qZmZWNLh3dADMza1uV/TaOWLm0aNuLpXPvjYhxRdtgETmomZmVuVi5\nlO5bHlK07X349B/WLdrGisxBzcys7AmUj7NN+ThKMzPLBWdqZmblToDU0a1oFw5qZmZ54PKjmZlZ\n5+JMzcwsD1x+NDOz8uDej2ZmZp2OMzUzszxw+dHMzMqCcPnRzMyss3GmZmZW9uTyo5mZlRGXH83M\nzDoXZ2pmZnng8qOZmZUHX3xtZmbW6ThTMzMrd370jJmZlRWXH83MzDoXZ2pmZmUvPx1FHNTMzPKg\nIh/n1PIRus3MLBecqZmZlTvfpd/MzKzzcaZmZpYHvk7NzMzKQ356P+bjKM3MLBecqZmZ5YHLj2Zm\nVjZcfjQzM+tcnKmZmZU7yeVHMzMrIy4/mpmZdS4OamVI0nRJezQybw9Js5pYd4Kk89qscWbWMepK\nkMUYSpiDWicj6XVJezWY9jVJ/64bj4hREfFguzeuCQ3bWOok7Svp35LmS3pL0hWS+rZw3eGSQtKi\nguGZIrTpHEnXre12ikXSFpJuljRP0gJJz0o6XVJlG++32S9ekk6RNEXSMkkTGszbRdL9kt6TNDcd\nw+CC+d+TNE3SQkmvSfpeGx1KO0oXXxdrKGGl3TqzIlGmNX/v/YHzgCHA1kAV8ItW7nZARPRJw3at\nXLfoJBXtHLqkzYDJwEzgExHRHzgYGAO0KPi3sdlkv78/r2beQOAyYDiwMbAQuKpgvoCj0nLjgFMk\nHdaWjbXicVArQ4XZnKSe6Zvt+5KeB3ZssOxoSVPTt9IbgR4N5u8n6emUsTwiadsG+zkjfUNfIOlG\nSaus38L2HiPphdSGVyV9o2DeNEn7F4x3TZnB6DS+S2rXfEnPFJZdJT0oabykh4ElwKYpY3y14Fv4\n4atrU0T8NSLuiYglEfE+cDmwW2uPrZHjPTYd7/uS7pW0ccG830qaKekDSU9K+lSaPg74EXBoYebX\nMHMvzOYKMsbjJL0J/G8L3rMWvT/AucAjEXF6RMxJ79mLEXF4RMxP2zpAWSl8fvpdbF2wn5A0omC8\nPvtSKpFL+q6kdyTNkXRMmncCcDjw/fQ+3LG6xkXE3yLiNuDd1cybFBE3R8QHEbEEuJiC321EXBgR\nUyNiZUS8CNxOkX73HcrlRysTPwU2S8PewNF1MyR1A24DrgUGATcDXyqYP5rsm+43gHWAPwETJXUv\n2P4hZN9mNwG2Bb62Bm18B9gP6AccA/xa0g5p3jXAEQXLfh6YExFPSaoC7iL7Rj4IOAO4VdJ6Bcsf\nCZxAlj3MBX4H7BMRfYFdgafTsW6UPnw3aqSNuwPT1+DYViHpQLLg9EVgPeAh4PqCRZ4Atk/H81fg\nZkk9IuIe4OfAjWuQ+X2aLNvcu6n3TFJvGnl/VmMv4JYmjnOLdFynpeO8G7gj/c21xIZk2XIVcBzw\nB0kDI+Iy4C/Ahel92D/t7xJJl7Rw2w01+ruVJOBTjc3vNOoePePyo5Wo29IH8HxJ84Gm/jMfAoyP\niPciYibZh1adXYCuwG8iYkVE3EL2oVrnBOBPETE5Imoi4mpgWVqvzu8iYnZEvAfcQfaB3CoRcVdE\nvBKZ/wPuI/sgAbgO+Lykfmn8SLIgDFmwuzsi7o6I2oi4H5hCFvjqTIiI6RGxElgJ1ALbSOoZEXMi\nYnpqw5sRMSAi3mzYPkmfJfsy8JNWHtq8gt/TGWnaicD5EfFCatPPge3rsrWIuC4i3k1Zwi+B7sCW\nrdxvQ+dExOKIWErz79lq35/VWAeY08Q+DwXuioj7I2IFcBHQkyxQtsQK4Gfp7/JuYBFNvA8R8c2I\n+GYLt10vVR5+AjR23uwcss/JqxqZbyXGQa1zOih9AA+IiAFAU/+Zh5Cd96jzRoN51RERjczfGPhu\ngwA6LK1X562C10uAPq05EABJ+0h6TNmJ+/lkH7DrAkTEbOBh4EuSBgD7kH1Tr2vfwQ3a90lgcMHm\n6489IhaTfdieCMyRdJekrZpp2y5kGdOXI+KlVh7augW/p4sK2vzbgva+R/Y9uirt74xUmlyQ5vev\ney/WQuHvv9H3rJXvz7us+j43NISCv6WIqE3tqGphm99NQb/OGv1tNSWVPycB346Ih1Yz/xSyc2v7\nRsSyYu67/bmjiJWPOWSBqM5GDeZVpRLL6ubPJMvyBhQMvSKisFy2VlIp81ayb/IbpCB9N9kHfZ2r\nyTKMg4FHI6K6oH3XNmhf74i4oGDdwoBNRNwbEZ8l+0D+D9m5ssbaNhqYCBwbEQ+s1YF+ZCbwjQZt\n7hkRj6TzZ98ny64HpvdiAR+9F7Ga7S0GehWMb7iaZQrXa/I9a8X78w8KStWrMZssgAL1ZbxhQN3v\nbkkL2t2Y1b0PrZIy438A/x0R165m/rHAmcCeEdHoJTCdis+pWZm4CfihpIGShgLfKpj3KFlJ7lRl\nHTC+COxUMP9y4ERJOyvTW1lX9zXt3SZJPQoHoBtZiW0usFLSPsDnGqx3G7AD8G2yc2x1rgP2l7S3\npMq0zT3Sca5u5xtIOjCdO1pGVtKqbWTZbYB7gG9FxMc6IyjrkPFgK469zqVkv49RaTv9JR2c5vUl\n+33MBbpI+gnZecY6bwPDtWovzqeBw9Lvbyzw5Wb23+h71pr3h+xc7a6SfiFpw3QsIyRdlzLqm4B9\nJe0pqSvw3bTNRwra/dXUhnFk5/1a6m1g06YWkNQl/X1VAnXH2SXNqyLrNHNxRFy6mnUPJysLfzYi\nXm1Fu6wEOKiVv3PJykCvkZ2rqv9WGhHLyTosfI2sDHYo8LeC+VOA48l6h70PzGDNOoLU2RVYuprh\nVLIPwfeBr5JlR/XSuaBbyTqjFLZvJlDX8WIuWRbyPRr/u64ATifLIt4j+yA9Ceo7iiwq6CjyXbIO\nDlfqo2vNCs8vDSMri7ZKRPwd+B/gBkkfANPISqoA95IF0pfIfmcfsmrp8Ob0811JU9Prs8k6Ab1P\n9rv+azP7b+o9a/T9Wc12XgH+H1m3+OmSFpD9jqYAC1OvwSOA3wPzgP2B/dPfHGRfUPYH5pP1Zryt\nqXY3cCUwMpVPbwOQdKmkwgB1Ftnf1pmpHUvTNICvkwXFcwp+t4sK1j2P7JzhEwXzPxb8Op2clB+1\n6ukUs9KUspYtIuKIZhduB5KeJitNfazLuFmpqRiwcXTf48dF296Ht3/jyYgYW7QNFpFvaGwlT9Ig\nsm7dR3Z0W+pERKt7eZpZ2yvtPNJyT9LxZCWySRHxr45uj1mnpPz0fnSmZiUtIi6niR6KZtZCJd5r\nsVhKO+SamZm1gjM1M7McUE4yNQe1NaQuPUPdSuFm5NaZjN66sVtLmq3eG2+8zrx589YqIgkHNWuG\nuvWl+5aHdHQzrJN5ePLFHd0E62R227kke86XLAc1M7NyJ1a98VwZc1AzMyt7yk350b0fzcysbDhT\nMzPLgbxkag5qZmY5kJeg5vKjmZmVDWdqZmY5kJdMzUHNzKzc5ahLv8uPZmZWNhzUzMzKnNJ1asUa\nmt2f9B1J0yVNk3S9pB6SBkm6X9LL6efAguV/KGmGpBcl7V0wfYyk59K836kFO3dQMzPLgfYKapKq\ngFOBsRGxDVAJHAacCTwQEZsDD6RxJI1M80cB44BLJFWmzf0ROB7YPA3jmjtOBzUzMyu2LkBPSV2A\nXsBs4EDg6jT/auCg9PpA4IaIWBYRrwEzgJ0kDQb6RcRjERHANQXrNLljMzMrc+3V+zEiqiVdBLwJ\nLAXui4j7JG0QEXPSYm8BG6TXVcBjBZuYlaatSK8bTm+SMzUzsxwocvlxXUlTCoYTCvYzkCz72gQY\nAvSWdERhW1LmFW1xnM7UzMysteZFRGPPxNkLeC0i5gJI+huwK/C2pMERMSeVFt9Jy1cDwwrWH5qm\nVafXDac3yZmamVm5U5GHpr0J7CKpV+qtuCfwAjARODotczRwe3o9EThMUndJm5B1CHk8lSo/kLRL\n2s5RBes0ypmamVkOtOM5tcmSbgGmAiuBp4DLgD7ATZKOA94ADknLT5d0E/B8Wv7kiKhJm/smMAHo\nCUxKQ5Mc1MzMrKgi4qfATxtMXkaWta1u+fHA+NVMnwJs05p9O6iZmZU55eghoQ5qZmY5kJeg5o4i\nZmZWNpypmZnlQT4SNQc1M7OyJ5cfzczMOh1namZmOZCXTM1BzcwsB/IS1Fx+NDOzsuFMzcyszPni\nazMzKy/5iGkuP5qZWflwpmZmVu5ydJ2ag5qZWQ7kJai5/GhmZmXDmZqZWQ7kJVNzUDMzy4N8xDSX\nH83MrHw4UzMzywGXH83MrCxI+bmjiMuPZmZWNpypmZnlQF4yNQc1M7McyEtQc/nRzMzKhjM1M7M8\nyEei5qBmZpYHLj+amZl1Ms7UzMzKnR89Y2Zm5UJATmKay49mZlY+nKmZmZW9/Nwmy0HNzCwHchLT\nXH40M7Py4UzNzCwH8lJ+dKZmZmZlw5mamVm5U37OqTmomZmVOQEVFfmIai4/mplZ2XCmZmaWAy4/\nmplZ2XDvRzMzs07GmZqZWblz70czMysX2V368xHVXH60VXx216155u9nM+32n3LGMZ/92PwBfXty\n4y+P5/Ebf8hD157ByM0G1887+St7MOXmH/HkLT/mlK/uscp6Jx32aZ7+21k8ecuPGf/tAwHo2qWS\nP51zBE/c9CMm33gmnxqzeZsem7Wd++69h21HbcmorUbwiwsv+Nj8iOD0005l1FYj2HH0tjw1dWqz\n657707PZcfS27Dxme/bb53PMnj0bgHfffZe99/ov1h3Qh9NOPaXtD846FQc1q1dRIX5z5iEceMol\njP7SeRw8bgxbbbrhKst8/7i9eebFWex06Pkcd/a1XPS9LwMwcrPBHPPFXfnUkb9gp0PPZ5/dt2HT\nYesCsPvYzdlvj0+w06EXMObL4/nNNQ8AcOwXdwNgx0N+zn4nXswFp38hN98my0lNTQ2nnXoyt98x\niaeefZ6bb7ieF55/fpVl7r1nEq/MeJlpL7zMxX+8jFNPOanZdb/z3e/xxFPPMvnJp9nn8/tx/nk/\nA6BHjx785Jz/5vz/uah9D7RTy+7SX6yhlDmoWb0dtxnOKzPn8Xr1u6xYWcPN905lvz22XWWZrTbd\nkP974iUAXnr9bTYeMoj1B/Vlq0025Ilpr7P0wxXU1NTy0JMzOOgz2wNwwsGf4qKr7mf5ipUAzH1/\nUf22HnzixfppCxYuZczIjdrrcK1Innj8cTbbbASbbLop3bp14+BDD+POO25fZZk7J97OV484Ckns\nvMsuLFgwnzlz5jS5br9+/erXX7Jkcf2Hae/evdntk5+kR48e7XeQZUAq3lDKHNSs3pD1+zPr7ffr\nx6vffp+q9fqvssxzL1Vz4Ge2A2DsqI3ZaPAgqjYYwPRXZrPb6BEM6t+bnj26Mu6Toxi64UAARmy8\nPruN3ox/XXMG913x7frA9dxL1ez36U9QWVnBxkPWYfTIYfXrWOcxe3Y1Q4cOqx+vqhpKdXV1s8vM\nrq5udt2fnv1jRmwyjBuu/wtnn/OzNjwKKxcOatYqF111P/379uKxG87kpMM+zTMvzqKmppYXX3ub\nX064nzsuOZmJfzi5fjpAl8oKBvXvze5HXcSPfn0b1114LABX3/4o1W/P5+G/fJ9ffO9LPPbMa/Xr\nmAGc+9/jmfHaTA77yuFcesnFHd2cTi0v5cd26/0o6ZGI2HUN1tseeArYJyLuSdMGAF+NiEvS+HBg\n14j46xq27UHgjIiYsibrl4vZ7yxg6AYfZUpVGwykeu6CVZZZuPhDvnHOdfXj/7nrXF6rfheAq297\nlKtvexSAc0/Zn+q35wNQ/fZ8bnvgaQCmTH+D2tpg3YF9mPf+Ir7/y7/Vb+ufE07n5TffaZuDszYz\nZEgVs2bNrB+vrp5FVVVVs8sMqapixYoVza4LcOhXDucLB3yes396bhscQQ50grJhsbRbprYmAS35\nCvDv9LPOAOCbBePDga+u4fYtmTL9DUZstB4bD1mHrl0qOXjvHbjrwWdXWaZ/n5507VIJwDFf2JV/\nT53BwsUfArDewD4ADNtwIAd+ZjtunJR9R7jjwWf59I5bADBio/Xp1rUL895fRM8eXenVoxsAn9l5\nK1bW1PKfV99ql2O14hm7447MmPEyr7/2GsuXL+fmG29g3/0OWGWZffc/gL9edw0RweTHHqNfv/4M\nHjy4yXVnvPxy/fp3TrydLbbcql2Pyzqn9szUFkVEH0mDgRuBfmn/J0XEQ42sI+Bg4LPAQ5J6RMSH\nwAXAZpKeBu4HPgVsncavBv4OXAv0Tps6JSIeSdv8AXAEUAtMiogzC/ZXAfwZmBURZxX3HSh9NTW1\nfOd/buKOS06mskJcfftjvPDqW3z9y58E4Ipb/s1Wm27I5T87kojghVfmcOK5f6lf//qLvs6gAb1Z\nsbKG0y64iQWLlgJZBvencw5nys0/YvmKGr7+k2sBWG9gX+645GRqa4PZc+dz3FlXt/9B21rr0qUL\nv/7txey/797U1NRw9NeOZeSoUVz+p0sBOP4bJzJun89z76S7GbXVCHr17MWfrriqyXUBzvrxmbz8\n0otUqIKNNt6Y3/3h0vp9bjliOAs/+IDly5dzx8TbuPPu+9h65Mj2P/hOIk/XqSki2mdHHwW17wI9\nImK8pEqgV0QsbGSd3YCfRcSekv4K3BoRt6Zy450RsU1abg+y8uF+abwXUBsRH0raHLg+IsZK2gc4\nG9grIpZIGhQR76Xy45nAt4FpETG+kfacAJwAQNc+Y3qMOroo743lx/tP+LyQtc5uO4/lySenrFVE\n6l21ZWx90qXNL9hCT579mScjYmzRNlhEHdFR5AngGEnnAJ9oLKAlXwFuSK9vYNUSZFO6ApdLeg64\nGaj7CrcXcFVELAGIiPcK1vkTTQS0tPxlETE2IsaqS88WNsXMzNpLuwe1iPgXsDtQDUyQdNTqlktZ\n3JeAn0h6Hfg9ME5S3xbs5jvA28B2wFigWwvWeQT4L0m++MXMyk5eej+2e1CTtDHwdkRcDlwB7NDI\nonsCz0bEsIgYHhEbA7cCXwAWAoXBreF4f2BORNQCRwKVafr9ZFlir9SWQQXrXAncDdwkyffENLOy\n4ouv284ewDOSngIOBX7byHJfIevwUehW4CsR8S7wsKRpkn4BPAvUSHpG0neAS4CjJT0DbAUsBkiX\nBEwEpqROJWcUbjwifkV2+cC1qdOImZl1Iu2WkUREn/TzarIeis0tf8xqpk0kC0pERMMu/J9pMF54\nf6cfFGzjArLek4Xb3aPg9U+ba5uZWaei/PR+dJnNzKzMZV36O7oV7aMkgpqkyUD3BpOPjIjnOqI9\nZmbWOZVEUIuInTu6DWZm5av0ey0WS0kENTMza1s5iWm+S7+ZmZUPZ2pmZjng8qOZmZWHTnDRdLG4\n/GhmZmXDmZqZWZnL06NnHNTMzHIgL0HN5UczMysbztTMzHIgJ4mag5qZWR64/GhmZtbJOKiZmZW7\nIj4gtKUJn6QBkm6R9B9JL0j6f5IGSbpf0svp58CC5X8oaYakFyXtXTB9jKTn0rzfqZmU00HNzKzM\nKd3QuFhDC/0WuCcitgK2A14AzgQeiIjNgQfSOJJGAocBo4BxwCWSKtN2/ggcD2yehnFN7dRBzczM\nikpSf2B34EqAiFgeEfOBA/noIdFXAwel1wcCN0TEsoh4DZgB7CRpMNAvIh6LiACuKVhntRzUzMxy\noJ3Lj5sAc4GrJD0l6QpJvYENImJOWuYtYIP0ugqYWbD+rDStKr1uOL1RDmpmZjlQIRVtANaVNKVg\nOKHB7roAOwB/jIjRwGJSqbFOyryi2MfpLv1mZtZa8yJibBPzZwGzImJyGr+FLKi9LWlwRMxJpcV3\n0vxqYFjB+kPTtOr0uuH0RjlTMzPLgfYsP0bEW8BMSVumSXsCzwMTgaPTtKOB29PricBhkrpL2oSs\nQ8jjqVT5gaRdUq/HowrWWS1namZmZS4LRu1+8fW3gL9I6ga8ChxDlkjdJOk44A3gEICImC7pJrLA\ntxI4OSJq0na+CUwAegKT0tAoBzUzMyu6iHgaWF2Jcs9Glh8PjF/N9CnANi3dr4OamVkOVOTjLlkO\namZmeeB7P5qZmXUyztTMzHIgJ4mag5qZWbkT2f0f88DlRzMzKxvO1MzMcsC9H83MrDy07pExnZrL\nj2ZmVjacqZmZ5UBOEjUHNTOzcieoe2RM2XP50czMyoYzNTOzHMhJouagZmaWB+79aGZm1sk4UzMz\nK3MtfWJ1OXBQMzPLAfd+NDMz62QazdQk9WtqxYj4oPjNMTOztpCPPK3p8uN0IFj1vagbD2CjNmyX\nmZkVUV56PzYa1CJiWHs2xMzMbG216JyapMMk/Si9HippTNs2y8zMiiW7TVbxhlLWbFCTdDHwX8CR\nadIS4NK2bJSZmRVRevRMsYZS1pIu/btGxA6SngKIiPckdWvjdpmZmbVaS4LaCkkVZJ1DkLQOUNum\nrTIzs6Iq8QSraFoS1P4A3AqsJ+lc4BDg3DZtlZmZFVWplw2LpdmgFhHXSHoS2CtNOjgiprVts8zM\nzFqvpbfJqgRWkJUgfRcSM7NOpK73Yx60pPfjj4HrgSHAUOCvkn7Y1g0zM7Pice/HjxwFjI6IJQCS\nxgNPAee3ZcPMzMxaqyVBbU6D5bqkaWZm1kmUdn5VPE3d0PjXZOfQ3gOmS7o3jX8OeKJ9mmdmZmtL\nys+jZ5rK1Op6OE4H7iqY/ljbNcfMzGzNNXVD4yvbsyFmZtZ2cpKoNX9OTdJmwHhgJNCjbnpEbNGG\n7TIzM2u1llxzNgG4iuw84z7ATcCNbdgmMzMrsrx06W9JUOsVEfcCRMQrEXEWWXAzM7NOQireUMpa\n0qV/Wbqh8SuSTgSqgb5t2ywzM7PWa0lQ+w7QGziV7Nxaf+DYtmyUmZkVj5C79NeJiMnp5UI+elCo\nmZl1Fp2gbFgsTV18/XfSM9RWJyK+2CYtMjMzW0NNZWoXt1srOqHRW2/Ew5P9FlnrTJu5oKObYJ3M\n0uU1RdlOqfdaLJamLr5+oD0bYmZmbScvzwzLy3GamVkOtPQhoWZm1kkJlx8/RlL3iFjWlo0xM7O2\n4SdfJ5J2kvQc8HIa307S79u8ZWZmZq3UknNqvwP2A94FiIhngP9qy0aZmVlxVah4QylrSfmxIiLe\naFCPLU4fUzMza3PZPRtLPBoVSUuC2kxJOwEhqRL4FvBS2zbLzMys9VoS1E4iK0FuBLwN/CNNMzOz\nTqLUy4bF0pJ7P74DHNYObTEzszaSk+pji558fTmruQdkRJzQJi0yMzNbQy0pP/6j4HUP4AvAzLZp\njpmZFZvAj56pExE3Fo5Luhb4d5u1yMzMii4v90Rck+PcBNig2A0xMzNbWy05p/Y+H51TqwDeA85s\ny0aZmVlx5aT62HRQU3a13nZAdZpUGxGNPjjUzMxKj6TcnFNrsvyYAtjdEVGTBgc0MzMrWS05p/a0\npNFt3hIzM2sz2a2yijOUskbLj5K6RMRKYDTwhKRXgMVkvUMjInZopzaamdla8h1F4HFgB+CAdmqL\nmZnZWmkqqAkgIl5pp7aYmVkb8MXXmfUknd7YzIj4VRu0x8zM2kBOYlqTQa0S6EPK2MzMzEpdU0Ft\nTkT8rN1aYmZmbaMTPLG6WJo9p2ZmZp2fcvKR3tR1anu2WyvMzMyKoNFMLSLea8+GmJlZ28h6P3Z0\nK9pHS56nZmZmnVxeglpeHrFjZmY54KBmZpYDkoo2tHB/lZKeknRnGh8k6X5JL6efAwuW/aGkGZJe\nlLR3wfQxkp5L836nFuzcQc3MrMzVnVMr1tBC3wZeKBg/E3ggIjYHHkjjSBoJHAaMAsYBl0iqTOv8\nETge2DwN45rbqYOamZkVlaShwL7AFQWTDwSuTq+vBg4qmH5DRCyLiNeAGcBOkgYD/SLisfTYs2sK\n1mmUO4qYmZW79n9kzG+A7wN9C6ZtEBFz0uu3gA3S6yrgsYLlZqVpK9LrhtOb5KBmZpYDRb6h8bqS\nphSMXxYRlwFI2g94JyKelLTH6laOiJDUJg+ddlAzM7PWmhcRYxuZtxtwgKTPAz2AfpKuA96WNDgi\n5qTS4jtp+WpgWMH6Q9O06vS64fQm+ZyamVmZa8+OIhHxw4gYGhHDyTqA/G9EHAFMBI5Oix0N3J5e\nTwQOk9Rd0iZkHUIeT6XKDyTtkno9HlWwTqOcqZmZ5UAJPHrmAuAmSccBbwCHAETEdEk3Ac8DK4GT\nI6ImrfNNYALQE5iUhiY5qJmZWZuIiAeBB9Prd2nknsIRMR4Yv5rpU4BtWrNPBzUzs7InKnJyl34H\nNTOzMidKovzYLtxRxMzMyoYzNTOzcucnX5uZWTkp8sXXJcvlRzMzKxvO1MzMylyeOoo4qJmZ5YDL\nj2ZmZp2MMzUzsxzISaLmoGZmVu5EfspyeTlOMzPLAWdqZmblTqCc1B8d1MzMciAfIc3lRzMzKyPO\n1MzMylz25Ot85GoOamZmOZCPkObyo5mZlRFnamZmOZCT6qODmplZ+VNuuvS7/GhmZmXDmZqZWZnL\n022yHNTMzHLA5UczM7NOxpmamVkO5CNPc6ZmDdx37z1sO2pLRm01gl9ceMHH5kcEp592KqO2GsGO\no7flqalTW7zub379S3p2FfPmzQPggX/cz647jWHs9p9g153G8OA//7ftDszaVL+eXdimqg/bDO3D\nhv27f2x+ZQVstn4vRlb1YeshvenR9aOPnvX7dWNUVR9GVfVh/X7d6qcPGdCdbYf1ZeSQPowc0of+\nPbPv4ALz1IaWAAAWsklEQVSGr9uTkWmd1e3PGkg3NC7WUMqcqVm9mpoaTjv1ZO6adD9VQ4fyyV12\nZL/9DmDrkSPrl7n3nkm8MuNlpr3wMo9Pnsypp5zEQ49MbnbdmTNn8sD99zFso43qt7XOOutyy213\nMGTIEKZPm8b+++7Nq29Ut/tx29rbaJ0evPTWYlasDLYe0of5S1bw4Yra+vmD+/dgyfIaXnlnCT26\nVrDROj156a3F9OhawXp9u/HC7EXUBmyxYW8WLFnJspXZum8vWMbbHyxfZV8De3elQvB89SIqBKOq\n+vLe4uUsXxntesxWmpypWb0nHn+czTYbwSabbkq3bt04+NDDuPOO21dZ5s6Jt/PVI45CEjvvsgsL\nFsxnzpw5za77/TO+w/jzL1zlW972o0czZMgQAEaOGsWHS5eybNmy9jlYK5re3StZtqKW5SuDAN5b\nvIIBvbquskyPbhUsXLoSgA9X1NKti+hSIXp2rWDRshpqUzxa+OFKBvZu/rt2RUX2dySJIKipbWaF\nnKvr/VisoZSVevusHc2eXc3QocPqx6uqhlJdXd3sMrOrq5tc946JtzNkSBXbbrddo/v++99uZfvR\nO9C9u0tJnU23SrG85qMsaXlNFrQKLV1ew8DeWaDr3a2S7l0q6NZFLF1RS98elVRWiApB/55d6FpZ\nUJrs352RVX0Yvm5P6ia/v3gFtbXBdhv1ZdthfXlrwTJqap2lNcflR7MiWLJkCRde8HPunHRfo8s8\nP306Z/3oB9x5d+PLWOc2Z/4yNlqnJyOH9GHpihqWLK8hyLK2t+YvY4sNe1MbwZLlNfXrvPPBcmbP\nzzL3qoHdGTaoJ6/PW0rv7pUE8OybC6msEFsN7s0HS1e6/GhAG2Zqkh5Zg3Vel3RrwfiXJU0oasOa\nb8M5ks5oz32WiiFDqpg1a2b9eHX1LKqqqppdZkhVVaPrvvrKK7zx+mvsNGY7thwxnOpZs/h/O+3A\nW2+9BcCsWbM49OAvcMWfr2HTzTZr4yO0trC8JuhW+dG3926VFR8LMLUBr89byvOzF/Ha3KV0qahg\nWTrnNm/RCl6YvYgX5yympjbqz8WtLMi+5i5cTu/ulQAM6tOVBUtXEmmZRctq6N3d38+boyIOpazN\nglpE7LqGq46RNLL5xT5Okv+y18LYHXdkxoyXef2111i+fDk333gD++53wCrL7Lv/Afz1umuICCY/\n9hj9+vVn8ODBja67zSc+wZuz3+HFGa/z4ozXqRo6lEcfn8qGG27I/Pnz+eIB+/Lf4y9g191266Cj\ntrW1eFkNPbpW0q2LEDCod1fmL1mxyjKVFR99GK7btysLP1xZfx6tSzo/1q1SDOjVlfcWZx1DuhYE\nyoG9urI0ZXHLV9bSr0f2X71C2Tm9DwsyPMu3NgsCkhZFRB9Jg4EbgX5pfydFxENNrPpL4MfA4Q22\nNwj4M7ApsAQ4ISKelXQOsFma/qake4GDgN7A5sBFQDfgSGAZ8PmIeE/S8cAJad4M4MiIWNLMMZ2Q\n1lmlF1+56NKlC7/+7cXsv+/e1NTUcPTXjmXkqFFc/qdLATj+Gycybp/Pc++kuxm11Qh69ezFn664\nqsl1m3LpJRfzyiszOP+8n3H+eT8D4I5J97H++uu37YFa0b357lK22LA3AO8uzHo+rtc3654/d+Fy\nenStZJP1ekLA0hW1vD7vo/9qm23Qiy4VIiLbTl2nj6GDetCzW5adLV9RyxvvLgWysuTw9XoxqqoP\nAPMWLWfpCvcUaU6JnworGkW0TR26IKh9F+gREeMlVQK9ImJhI+u8DuwMPAjsD2wP7BcRX5P0e2Be\nRJwr6TPAryJi+xTU9gc+GRFLJX0NOAsYDfQgC1g/iIhLJf0aeCMifiNpnYh4N+33PODtiPh92t6i\niLioqeMbM2ZsPDx5ytq8RZZD02Yu6OgmWCdz1AF78PxzT61VSNp81HbxqxuKd876gG03fDIixhZt\ng0XUHr0fnwCOScHiE40FtAI1wC+AHzaY/kngWoCI+F9gHUn90ryJEbG0YNl/RsTCiJgLLADuSNOf\nA4an19tIekjSc2RZYdNphZmZlbw2D2oR8S9gd6AamCDpqBasdm1aZ1hzCyaLG4wXXuxUWzBey0cl\n1wnAKRHxCeBcsqzOzKwsScUbSlmbBzVJG5OV9i4HrgB2aG6diFgB/Br4TsHkh0jn2STtQVaK/GAt\nmtYXmCOpKw3O35mZlRcV9V8pa4/egnsA35O0AlgEtCRTA7iS7NxYnXOAP0t6lqyjyNFr2a6zgcnA\n3PSz71puz8zMOlibBbWI6JN+Xg1c3cJ1hhe8XgYMKRh/j6xXY8N1zmkwPoGstLi6bdbPi4g/An9s\nbntmZuWg1MuGxeLruszMylx278d8RLUOCWqSJgMNb/J3ZEQ81xHtMTOz8tAhQS0idu6I/ZqZ5VIn\n6LVYLC4/mpnlQF6Cmh89Y2ZmZcOZmplZDpT69WXF4qBmZlbmRPZEgzxw+dHMzMqGMzUzsxxw+dHM\nzMqGez+amZl1Ms7UzMxywOVHMzMrC+79aGZm1gk5UzMzK3ul/3DPYnFQMzMrdzm6obHLj2ZmVjac\nqZmZ5UBOEjUHNTOzcpf1fsxHWHP50czMyoYzNTOzHMhHnuagZmaWDzmJai4/mplZ2XCmZmaWA774\n2szMykZOOj+6/GhmZuXDmZqZWQ7kJFFzUDMzy4WcRDWXH83MrGw4UzMzK3PCvR/NzKxc+NEzZmZm\nnY+DmplZDqiIQ7P7koZJ+qek5yVNl/TtNH2QpPslvZx+DixY54eSZkh6UdLeBdPHSHouzfud1HTO\n6aBmZpYH7RnVYCXw3YgYCewCnCxpJHAm8EBEbA48kMZJ8w4DRgHjgEskVaZt/RE4Htg8DeOa2rGD\nmpmZFVVEzImIqen1QuAFoAo4ELg6LXY1cFB6fSBwQ0Qsi4jXgBnATpIGA/0i4rGICOCagnVWyx1F\nzMzKnjqs96Ok4cBoYDKwQUTMSbPeAjZIr6uAxwpWm5WmrUivG05vlIOamVkOFLn347qSphSMXxYR\nl318n+oD3AqcFhEfFJ4Oi4iQFEVtFQ5qZmbWevMiYmxTC0jqShbQ/hIRf0uT35Y0OCLmpNLiO2l6\nNTCsYPWhaVp1et1weqN8Ts3MrMwVs49IC3s/CrgSeCEiflUwayJwdHp9NHB7wfTDJHWXtAlZh5DH\nU6nyA0m7pG0eVbDOajlTMzPLg/Y9pbYbcCTwnKSn07QfARcAN0k6DngDOAQgIqZLugl4nqzn5MkR\nUZPW+yYwAegJTEpDoxzUzMysqCLi3zQeRvdsZJ3xwPjVTJ8CbNPSfTuomZnlgO/9aGZmZcP3fjQz\nM+tknKmZmeVAThI1BzUzs7LX8ns2dnouP5qZWdlwpmZmlgPu/WhmZmVBuPejmZlZp+NMzcwsB3KS\nqDmomZnlQk6imsuPZmZWNpypmZnlgHs/mplZ2XDvRzMzs07GmZqZWQ7kJFFzUDMzy4WcRDWXH83M\nrGw4UzMzK3PZTfrzkao5qJmZlTu596OZmVmn40zNzCwHcpKoOaiZmeVCTqKag9oamjr1yXk9u+qN\njm5HCVoXmNfRjbBOx383jdu4oxvQmTioraGIWK+j21CKJE2JiLEd3Q7rXPx309bk3o9mZlY+3PvR\nzMysk3GmZsV2WUc3wDol/920IZGbfiIOalZcEeEPJ2s1/920g5xENZcfzcysbDhTMzPLgbz0fnSm\nZmZmZcOZmpUMSYqI6Oh2WOmTNAhYNyJe6ui2dBbu0m/WTiQNA3BAs5aQ1AM4FThW0tYd3Z7OQkUc\nSpmDmrU7SX0kdUuvtwYulNS3g5tlnUREfAj8I40eLGlkR7bHSouDmrUrSb2BvwAHp0lL0rBIUte0\nTKl/GbQOUve3ERH/BiYC/YAvO7A1Iz1PrVhDKXNQs3YVEYuBG4FjJB0KDAeWRmZFWsZlSPuYunOu\nkjaR1CUiHgGuAvqTBTaXIpuUjwKkO4pYu5FUGRE1EfFXSXOBHwBPAptI+i0wC1gGdImIX3VkW630\npIC2L3A28JCkRcBvyO5GchxwhKS/RMTzHdlO61jO1KxdpG/ZNZI+K+nCiLgf+C2wJ7AceDP97ANM\n7sCmWomStAvwc+BQsi/kBwEXAnOBq4HeZH9D1oDIT/nRmZq1i/Qte0/gEuAbadodklYCpwMvRcQd\nHdlGK02SKoAge+baUcBWwO7AmcAJwEVkWf+PU3nbVqPEY1HROFOzNqdMF2AccHZE/G9d78eImARc\nCvxAUlVHttNKS0GHoT7pnOudEfEMWYb29Yi4F3iH7Mv5Bg5oBg5q1g7SB9JK4ENgF0k9ImI5gKQd\ngbuBAyKiuiPbaaWl4BzaA5LOkfTFNGt94ARJOwM7ARdFxLQOa2gnkZfyo4OatYm6b9mSNpI0NE2e\nBHQFPp3mbQf8GtgiIt7rkIZayZI0GDicrLz4HrB3CnLHAsOAnwDnR8SzHdfKzkNF/FfKfE7N2kTB\nt+zzgUckDYqIQ1K36yMl/YCsK/Z5qaRkVk/SWGA7oDoibpS0HrA38AWga0TsJ6lXRCzx7dWskIOa\nFVXBtUS7kPVM248sM/uzpH9ExF6SJpB9YC2IiFf8oWSFJO1B1pvxXrJu+tdHxFRJk4BuwIGSHo+I\n2eDrGlustBOsonFQs6JI9+NbkbrtbwC8CxwCbE7W27E/8KCkRyJiV2Bq3br+ULI6kjYBfgQcGRH/\nkjQDuE7S4RHxlKTbgXvqApq1XE5ims+p2dpLXa53BU6TtB/ZuY6FwPPAvsCfI2Ih2bfvjVLnEDNg\nlfOvO5Jl9f3JejgSERcCVwITJY2JiHcd0KwpDmpWLM8CnwOuBW6JiLfIvhzOATaTdDxZKfKzEfFE\nxzXTSk0qV+9OVq5+juwC616STknzfwn8gezCfFsDxez56N6PVrYk9ZY0NCJqgY3T5H8C+6Ru+7Vk\nd1NfQhbQLo2IFzqouVaiJG0JnARMiIgngQeBB4CtJH0XICIuiIj/882u11xeej86qNnaGA78XtKP\ngTOA7wLfIrtzet29G18lC3Rfioi/+UPJVuMTwAbAXpLWi4gFwD3AI8CWkuq+MPn8qzXLQc3WWERM\nB2aQndifnC6AnUt2K6zukh4g+9a9Il187Q8lKzyHNlRS/4i4hewmxR+Q3W1/nXQO9g7gJxHxRgc2\nt3zk4yb97v1orSNpALA8IpakSdOAXwJHSXouIh4Ank3Z22eB2RHxWAc110qMpIqIqJW0D9k5tBcl\nrU/WMeROYB+y6xivjYh3yTocWRGUeCwqGgc1azFJg4CXgH9Ieigi/hARV6d5M4FfSToamA98se7x\nMb4OzST1jIilKaCNAP4b+EZEPCLpd8BtZBdXd00/e5NdFmLWKg5q1hrvA/eR9Wg8XNJOwL+BmyPi\ncknLgVuBlcBpdSs5oOWbpP7ABZL+HhH3kX3p+Q/ZFyQi4lRJ1wNnRsRPJT0REXM6sMllKS9ns31O\nzVosBaepZCf1dwcmpJ//J+m/yDqE7EzWKWRSR7XTSk4/snOvX02PH/oAWAfYq2CZu0nPQnNAawvF\n7PtY2tHRmZq1SkRcJOlusg+kacD2ZN+6DwNGAIf6jukGIKlvRCyMiJmSriH7GzmWrDPRj4AJkrYC\nFqTp3++41lq5cFCzFpNUGRE1ZBnaF8jusH9lCnTrk91odl5HttFKg6ThwC2SngRuAl4GrgKWkV36\n8T/AwWQdQ4YA34mIf/j8a9uoe/J1HjioWYulgAYwGTgHeDQiLkrT5vrDyAr0AAYDBwKvk90R5FJg\nINn1Z2cD4yPit4Ur+W/I1pbPqVmrpG/SbwCnA33qnlbtDyOrk7rt/4esRL0AeBM4FJhNdm/HL6fx\nCyUNSPcONSsKZ2r2MQWPj6lIt7qqVxC8ZgG1H1/b8i5126+IiBckHQHcAPw8Iq6UdAvZkxsOBJ6O\niPkd2tgccfnRcqkgoO1JlondGxEfNlwuIqZJ+kFEVHdAM63EFQS2JyQdBlyf7gf6B+BFsguvfQ1j\nOyr1XovF4rTf6qWOICFpHPBH4P3VBTRlKiLiDUm9JK3T/q21UlcY2MjKjWdLOrnBMg5oVlQOaoak\nEan7dY2kgWQn8U9MD2n8lKSj04XWdepudTSA7Nq0QR3ScCsJBfdy/NjnSUFgexLYH5je3u0zIEeP\nnnH50SC7mHp9SY9FxPuS/gkcl56BVgGsIDsP8rikLhGxMt0l4mbgexHxcsc13TpSS8rVDTI2lxw7\nQCe4D3HROFMzIuJhsoczviqpH9l1aI8Dv4+IQ8muMxolqVsKaAOBvwM/i4h/dVS7rWO1tFxdt3ha\npydZt36zNuGgZgCkR318m+waonkR8dt0s9lPkd189oqIWJ4W/wpwXkQ81EHNtQ7U2nJ13UX7qVz9\nINktsqy9+dEzljcRcbukFcCTksYAH5JdU3RWRNxVVzaKiEs6tqXWwVyu7oTy0vvRQc1WERF3S6oF\nXgC2BH4QER8WnDvx+ZCci4iHJfUlK1dvS1au3hd4ImX3BwDHpHL18pTN3Qr81Nm9tTWXH+1jIuIe\n4OvA6LpzJHWBzAHNwOXqzsi9Hy3XIuIucE81a5zL1Z1LiceionFQsyY5oFlTXK62UuPyo5mtFZer\nOwn3fjQzaxmXq0tfXno/OlMzs6JxQLOO5kzNzKzM5enJ1/IXKzOz8ibpHmDdIm5yXkSMK+L2isZB\nzczMyobPqVnZklQj6WlJ0yTdLKnXWmxrD0l3ptcHSDqziWUHSPrmGuzjHElntHR6g2UmSPpyK/Y1\nXNK01rbRrNQ5qFk5WxoR20fENsBy4MTCmXUPO23tRiNiYkRc0MQiA4BWBzUzW3sOapYXDwEjUoby\noqRrgGnAMEmfk/SopKkpo+sDIGmcpP9Imgp8sW5Dkr4m6eL0egNJf5f0TBp2BS4ANktZ4i/Sct+T\n9ISkZyWdW7CtH0t6SdK/yS5ebpKk49N2npF0a4Pscy9JU9L29kvLV0r6RcG+v7G2b6RZKXNQs7In\nqQuwD9kz4yC7g/wlETEKWAycBewVETsAU4DTJfUALid7WvMYYMNGNv874P8iYjtgB7InO58JvJKy\nxO9J+lza507A9sAYSbunW0sdlqZ9HtixBYfzt4jYMe3vBeC4gnnD0z72BS5Nx3AcsCAidkzbP17S\nJi3Yj1mn5C79Vs56Sno6vX4IuBIYArwREY+l6bsAI4GHlfV57gY8CmwFvFb3mBRJ1wEnrGYfnwGO\nAoiIGmBBuit9oc+l4ak03ocsyPUF/h4RS9I+JrbgmLaRdB5ZibMPcG/BvJsiohZ4WdKr6Rg+B2xb\ncL6tf9r3Sy3Yl1mn46Bm5WxpRGxfOCEFrsWFk4D7I+IrDZZbZb21JOD8iPhTg32ctgbbmgAcFBHP\nSPoasEfBvIZdmSPt+1sRURj8kDR8DfZtVvJcfrS8ewzYTdIIAEm9JW0B/AcYLmmztNxXGln/AeCk\ntG5lehjmQrIsrM69wLEF5+qqJK0P/As4SFLP9Hyy/VvQ3r7AHEldgcMbzDtYUkVq86bAi2nfJ6Xl\nkbSFpN4t2I9Zp+RMzXItIuamjOd6Sd3T5LMi4iVJJwB3SVpCVr7su5pNfBu4TNJxQA1wUkQ8Kunh\n1GV+UjqvtjXwaMoUFwFHRMRUSTcCzwDvAE+0oMlnA5OBuelnYZveBB4H+gEnprvlX0F2rm2qsp3P\nBQ5q2btj1vn44mszMysbLj+amVnZcFAzM7Oy4aBmZmZlw0HNzMzKhoOamZmVDQc1MzMrGw5qZmZW\nNhzUzMysbPx//1P/Yyn+jIIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb7b2617cc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = Train.actual_value, pred_value = Train.pred_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-03T13:31:23.808404Z",
     "start_time": "2017-06-03T13:31:23.777445Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#4.5 GB\n",
    "pd.Series(Train.pred_value).to_csv('LSTM_prediction_values-.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/7d1ace18a82178e15ece8fc5252fce88"
  },
  "anaconda-cloud": {},
  "gist": {
   "data": {
    "description": "Hyper parameter tuning",
    "public": false
   },
   "id": "7d1ace18a82178e15ece8fc5252fce88"
  },
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
